In [191]:
import pandas as pd
import numpy as np
from sklearn.ensemble import VotingClassifier
from sklearn.metrics import roc_curve, auc, precision_recall_curve, average_precision_score
from sklearn.preprocessing import label_binarize
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.neural_network import MLPClassifier
from xgboost import XGBClassifier
from lightgbm import LGBMClassifier
from sklearn.metrics import (
classification_report, roc_auc_score, roc_curve, precision_recall_curve, f1_score,
matthews_corrcoef, confusion_matrix, accuracy_score, precision_score, recall_score, auc
)
from sklearn.base import clone
from sklearn.model_selection import cross_val_score
from sklearn.metrics import auc as calculate_auc
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import GridSearchCV, StratifiedKFold, learning_curve
from sklearn.experimental import enable_halving_search_cv # 👈 REQUIRED to use HalvingGridSearchCV
from sklearn.model_selection import HalvingGridSearchCV
import shap
# Ignore warnings for clean output
import warnings
warnings.filterwarnings('ignore')
from sklearn.preprocessing import label_binarize
from sklearn.base import clone
In [2]:
X_train_multi = pd.read_csv(r"C:\Users\ghaza\Desktop\FYP\Application\AI\data\processed\balanced_X_train_multi.csv")
y_train_multi = pd.read_csv(r"C:\Users\ghaza\Desktop\FYP\Application\AI\data\processed\balanced_y_train_multi.csv").squeeze()
X_test = pd.read_csv(r"C:\Users\ghaza\Desktop\FYP\Application\AI\data\processed\X_test.csv")
y_test_multi = pd.read_csv(r"C:\Users\ghaza\Desktop\FYP\Application\AI\data\processed\y_test_multi.csv").squeeze()
# Verify loaded data
print("Datasets Loaded:")
print(f"X_train_multi: {X_train_multi.shape}")
print(f"y_train_multi: {y_train_multi.shape}")
print(f"X_test: {X_test.shape}")
print(f"y_test_multi: {y_test_multi.shape}")
Datasets Loaded: X_train_multi: (330556, 7) y_train_multi: (330556,) X_test: (52487, 7) y_test_multi: (52487,)
In [3]:
# For y_train_multi
if isinstance(y_train_multi, pd.DataFrame) and y_train_multi.shape[1] == 1:
y_train_multi = y_train_multi.values.ravel()
elif isinstance(y_train_multi, pd.Series):
y_train_multi = y_train_multi.values
elif isinstance(y_train_multi, np.ndarray) and y_train_multi.ndim > 1:
y_train_multi = y_train_multi.ravel()
# For y_test_multi
if isinstance(y_test_multi, pd.DataFrame) and y_test_multi.shape[1] == 1:
y_test_multi = y_test_multi.values.ravel()
elif isinstance(y_test_multi, pd.Series):
y_test_multi = y_test_multi.values
elif isinstance(y_test_multi, np.ndarray) and y_test_multi.ndim > 1:
y_test_multi = y_test_multi.ravel()
# === Verification ===
print("Datasets Loaded and Verified:")
print(f"X_train_multi: {X_train_multi.shape}")
print(f"y_train_multi: {y_train_multi.shape}")
print(f"X_test: {X_test.shape}")
print(f"y_test_multi: {y_test_multi.shape}")
Datasets Loaded and Verified: X_train_multi: (330556, 7) y_train_multi: (330556,) X_test: (52487, 7) y_test_multi: (52487,)
In [8]:
models = {
"Decision Tree": DecisionTreeClassifier(random_state=42),
"Random Forest": RandomForestClassifier(random_state=42),
"XGBoost": XGBClassifier(objective="multi:softprob", num_class=5, use_label_encoder=False, eval_metric="mlogloss", verbosity=0),
"LightGBM": LGBMClassifier(objective="multiclass", num_class=5),
"Gradient Boosting": GradientBoostingClassifier(random_state=42),
"Logistic Regression": LogisticRegression(multi_class="multinomial", solver="lbfgs", max_iter=1000),
"MLP": MLPClassifier(max_iter=1000, random_state=42)
}
In [10]:
param_grid_xgb = {"n_estimators": [50], "max_depth": [3, 5], "learning_rate": [0.05, 0.1]}
param_grid_lgb = {"n_estimators": [50], "max_depth": [5, 10], "learning_rate": [0.05, 0.1]}
param_grid_dt = {"max_depth": [None, 5, 10], "min_samples_split": [2, 5]}
param_grid_rf = {"n_estimators": [50, 100], "max_depth": [None, 5, 10], "min_samples_split": [2, 5]}
param_grid_gb = {"n_estimators": [50, 100], "learning_rate": [0.05, 0.1], "max_depth": [3, 5]}
param_grid_lr = {"C": [0.1, 1.0, 10.0], "penalty": ["l2"]}
param_grid_mlp = {"hidden_layer_sizes": [(100,), (50, 50)], "activation": ["relu", "tanh"], "learning_rate_init": [0.001]}
In [12]:
models_with_grids = {
"Decision Tree": {"model": models["Decision Tree"], "param_grid": param_grid_dt},
"Random Forest": {"model": models["Random Forest"], "param_grid": param_grid_rf},
"XGBoost": {"model": models["XGBoost"], "param_grid": param_grid_xgb},
"LightGBM": {"model": models["LightGBM"], "param_grid": param_grid_lgb},
"Gradient Boosting": {"model": models["Gradient Boosting"], "param_grid": param_grid_gb},
"Logistic Regression": {"model": models["Logistic Regression"], "param_grid": param_grid_lr},
"MLP": {"model": models["MLP"], "param_grid": param_grid_mlp}
}
In [14]:
cv_strategy = StratifiedKFold(n_splits=3, shuffle=True, random_state=42)
In [16]:
def run_multiclass_grid_search(X_train, y_train, X_test, y_test, models_with_grids, cv_strategy, scoring="f1_macro"):
import pandas as pd
import numpy as np
from sklearn.metrics import (
classification_report, accuracy_score, f1_score, matthews_corrcoef,
confusion_matrix, precision_recall_curve, roc_auc_score, auc
)
from sklearn.model_selection import GridSearchCV
from itertools import product
grid_search_results = {}
trained_models = {}
threshold = 0.1 # 10% gap for over/under-fitting check
def check_over_under(train_acc, test_acc):
if train_acc > test_acc + threshold:
return "Potential Overfitting Detected"
elif test_acc > train_acc + threshold:
return "Potential Underfitting Detected"
else:
return "Balanced Performance"
def manual_search(model_class, param_grid, model_name):
print(f"\nRunning manual grid search for {model_name}...")
keys, values = zip(*param_grid.items())
best_score = -np.inf
best_model = None
best_params = None
for combo in product(*values):
params = dict(zip(keys, combo))
try:
model = model_class(**params)
model.fit(X_train, y_train)
preds = model.predict(X_test)
score = f1_score(y_test, preds, average="macro")
if score > best_score:
best_score = score
best_model = model
best_params = params
except Exception as e:
print(f"Skipping {params} due to error: {e}")
return best_model, best_params
for model_name, model_data in models_with_grids.items():
print(f"\nStarting grid search for {model_name}...")
model = model_data["model"]
param_grid = model_data["param_grid"]
if model_name in ["XGBoost", "LightGBM"]:
best_model, best_params = manual_search(model.__class__, param_grid, model_name)
else:
grid_search = GridSearchCV(
estimator=model,
param_grid=param_grid,
scoring=scoring,
cv=cv_strategy,
verbose=2,
n_jobs=-1
)
grid_search.fit(X_train, y_train)
best_model = grid_search.best_estimator_
best_params = grid_search.best_params_
trained_models[model_name] = best_model
print(f"Best Parameters for {model_name}: {best_params}")
y_train_pred = best_model.predict(X_train)
y_test_pred = best_model.predict(X_test)
train_acc = accuracy_score(y_train, y_train_pred)
test_acc = accuracy_score(y_test, y_test_pred)
fit_status = check_over_under(train_acc, test_acc)
print(f"Train Accuracy: {train_acc:.4f}")
print(f"Test Accuracy: {test_acc:.4f}")
print(f"{fit_status}")
f1 = f1_score(y_test, y_test_pred, average="macro")
mcc = matthews_corrcoef(y_test, y_test_pred)
cm = confusion_matrix(y_test, y_test_pred)
specificity = cm[0, 0] / np.sum(cm[0]) if cm.shape[0] > 1 else None
auc_roc = auc_pr = None
if hasattr(best_model, "predict_proba"):
try:
y_probs = best_model.predict_proba(X_test)
auc_roc = roc_auc_score(y_test, y_probs, multi_class='ovr')
auc_pr = np.mean([
auc(*precision_recall_curve((y_test == i).astype(int), y_probs[:, i])[1::-1])
for i in np.unique(y_test)
])
except Exception as e:
print(f"AUC calculation failed for {model_name}: {e}")
print(f"\nClassification Report for {model_name}:\n", classification_report(y_test, y_test_pred))
grid_search_results[model_name] = {
"Best Parameters": best_params,
"Train Accuracy": train_acc,
"Test Accuracy": test_acc,
"F1-Score": f1,
"AUC-PR": auc_pr,
"AUC-ROC": auc_roc,
"MCC": mcc,
"Specificity": specificity,
"Fit Status": fit_status
}
print(f"Results for {model_name}:")
print(grid_search_results[model_name])
results_df = pd.DataFrame(grid_search_results).T
print("\nSummary of All Model Results:")
print(results_df)
return trained_models, grid_search_results, results_df
In [18]:
trained_models, grid_results, summary_df = run_multiclass_grid_search(
X_train=X_train_multi,
y_train=y_train_multi,
X_test=X_test,
y_test=y_test_multi,
models_with_grids=models_with_grids,
cv_strategy=cv_strategy,
scoring="f1_macro"
)
Starting grid search for Decision Tree...
Fitting 3 folds for each of 6 candidates, totalling 18 fits
Best Parameters for Decision Tree: {'max_depth': None, 'min_samples_split': 2}
Train Accuracy: 1.0000
Test Accuracy: 0.7544
Potential Overfitting Detected
Classification Report for Decision Tree:
precision recall f1-score support
0 0.29 0.44 0.35 2683
1 0.95 0.82 0.88 38015
2 0.05 0.17 0.08 797
3 0.77 0.85 0.81 5512
4 0.35 0.44 0.39 5480
accuracy 0.75 52487
macro avg 0.48 0.54 0.50 52487
weighted avg 0.82 0.75 0.78 52487
Results for Decision Tree:
{'Best Parameters': {'max_depth': None, 'min_samples_split': 2}, 'Train Accuracy': 1.0, 'Test Accuracy': 0.7543772743727019, 'F1-Score': 0.5014763838607188, 'AUC-PR': 0.5377037520327916, 'AUC-ROC': 0.737595282931572, 'MCC': 0.5350155226346356, 'Specificity': 0.4383153186731271, 'Fit Status': 'Potential Overfitting Detected'}
Starting grid search for Random Forest...
Fitting 3 folds for each of 12 candidates, totalling 36 fits
Best Parameters for Random Forest: {'max_depth': None, 'min_samples_split': 2, 'n_estimators': 100}
Train Accuracy: 1.0000
Test Accuracy: 0.8104
Potential Overfitting Detected
Classification Report for Random Forest:
precision recall f1-score support
0 0.46 0.51 0.48 2683
1 0.95 0.87 0.91 38015
2 0.06 0.10 0.08 797
3 0.82 0.90 0.86 5512
4 0.41 0.57 0.48 5480
accuracy 0.81 52487
macro avg 0.54 0.59 0.56 52487
weighted avg 0.84 0.81 0.82 52487
Results for Random Forest:
{'Best Parameters': {'max_depth': None, 'min_samples_split': 2, 'n_estimators': 100}, 'Train Accuracy': 1.0, 'Test Accuracy': 0.8104101968106389, 'F1-Score': 0.5605885000054511, 'AUC-PR': 0.5823549331033362, 'AUC-ROC': 0.9107117889577276, 'MCC': 0.6203093295989356, 'Specificity': 0.506895266492732, 'Fit Status': 'Potential Overfitting Detected'}
Starting grid search for XGBoost...
Running manual grid search for XGBoost...
Best Parameters for XGBoost: {'n_estimators': 50, 'max_depth': 5, 'learning_rate': 0.1}
Train Accuracy: 0.7086
Test Accuracy: 0.7408
Balanced Performance
Classification Report for XGBoost:
precision recall f1-score support
0 0.51 0.54 0.53 2683
1 0.97 0.80 0.87 38015
2 0.05 0.56 0.09 797
3 0.80 0.90 0.85 5512
4 0.52 0.30 0.38 5480
accuracy 0.74 52487
macro avg 0.57 0.62 0.54 52487
weighted avg 0.86 0.74 0.79 52487
Results for XGBoost:
{'Best Parameters': {'n_estimators': 50, 'max_depth': 5, 'learning_rate': 0.1}, 'Train Accuracy': 0.7086030808698073, 'Test Accuracy': 0.7407929582563302, 'F1-Score': 0.544558698061361, 'AUC-PR': 0.5950468273043812, 'AUC-ROC': 0.9176916304982848, 'MCC': 0.5475955078822186, 'Specificity': 0.5415579575102497, 'Fit Status': 'Balanced Performance'}
Starting grid search for LightGBM...
Running manual grid search for LightGBM...
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.011710 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1785
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 7
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.009163 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1785
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 7
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.010699 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1785
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 7
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.009463 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1785
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 7
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
Best Parameters for LightGBM: {'n_estimators': 50, 'max_depth': 10, 'learning_rate': 0.1}
Train Accuracy: 0.7510
Test Accuracy: 0.7544
Balanced Performance
Classification Report for LightGBM:
precision recall f1-score support
0 0.49 0.55 0.52 2683
1 0.97 0.81 0.88 38015
2 0.05 0.44 0.09 797
3 0.81 0.91 0.86 5512
4 0.47 0.37 0.42 5480
accuracy 0.75 52487
macro avg 0.55 0.62 0.55 52487
weighted avg 0.86 0.75 0.80 52487
Results for LightGBM:
{'Best Parameters': {'n_estimators': 50, 'max_depth': 10, 'learning_rate': 0.1}, 'Train Accuracy': 0.7509862171613887, 'Test Accuracy': 0.75441537904624, 'F1-Score': 0.5515070952932931, 'AUC-PR': 0.5928606148635792, 'AUC-ROC': 0.9178655782468033, 'MCC': 0.561307202499785, 'Specificity': 0.554230339172568, 'Fit Status': 'Balanced Performance'}
Starting grid search for Gradient Boosting...
Fitting 3 folds for each of 8 candidates, totalling 24 fits
Best Parameters for Gradient Boosting: {'learning_rate': 0.1, 'max_depth': 5, 'n_estimators': 100}
Train Accuracy: 0.7718
Test Accuracy: 0.7542
Balanced Performance
Classification Report for Gradient Boosting:
precision recall f1-score support
0 0.47 0.55 0.51 2683
1 0.97 0.80 0.88 38015
2 0.05 0.38 0.09 797
3 0.80 0.91 0.85 5512
4 0.43 0.42 0.43 5480
accuracy 0.75 52487
macro avg 0.54 0.61 0.55 52487
weighted avg 0.85 0.75 0.80 52487
Results for Gradient Boosting:
{'Best Parameters': {'learning_rate': 0.1, 'max_depth': 5, 'n_estimators': 100}, 'Train Accuracy': 0.771838962233328, 'Test Accuracy': 0.7541867510050108, 'F1-Score': 0.5495046428954449, 'AUC-PR': 0.5896444483046593, 'AUC-ROC': 0.9150207845151066, 'MCC': 0.5603425328386388, 'Specificity': 0.5501304509877003, 'Fit Status': 'Balanced Performance'}
Starting grid search for Logistic Regression...
Fitting 3 folds for each of 3 candidates, totalling 9 fits
Best Parameters for Logistic Regression: {'C': 1.0, 'penalty': 'l2'}
Train Accuracy: 0.6714
Test Accuracy: 0.7140
Balanced Performance
Classification Report for Logistic Regression:
precision recall f1-score support
0 0.44 0.56 0.50 2683
1 0.96 0.77 0.85 38015
2 0.05 0.53 0.09 797
3 0.76 0.88 0.82 5512
4 0.43 0.28 0.34 5480
accuracy 0.71 52487
macro avg 0.53 0.60 0.52 52487
weighted avg 0.85 0.71 0.77 52487
Results for Logistic Regression:
{'Best Parameters': {'C': 1.0, 'penalty': 'l2'}, 'Train Accuracy': 0.6713658200123428, 'Test Accuracy': 0.7140053727589689, 'F1-Score': 0.5187632053437157, 'AUC-PR': 0.5698716080382211, 'AUC-ROC': 0.9038811531490225, 'MCC': 0.5145789572844949, 'Specificity': 0.5624301155423034, 'Fit Status': 'Balanced Performance'}
Starting grid search for MLP...
Fitting 3 folds for each of 4 candidates, totalling 12 fits
Best Parameters for MLP: {'activation': 'tanh', 'hidden_layer_sizes': (50, 50), 'learning_rate_init': 0.001}
Train Accuracy: 0.8277
Test Accuracy: 0.7533
Balanced Performance
Classification Report for MLP:
precision recall f1-score support
0 0.40 0.52 0.45 2683
1 0.96 0.80 0.87 38015
2 0.04 0.19 0.07 797
3 0.79 0.88 0.83 5512
4 0.35 0.50 0.41 5480
accuracy 0.75 52487
macro avg 0.51 0.58 0.53 52487
weighted avg 0.84 0.75 0.79 52487
Results for MLP:
{'Best Parameters': {'activation': 'tanh', 'hidden_layer_sizes': (50, 50), 'learning_rate_init': 0.001}, 'Train Accuracy': 0.8277266181826982, 'Test Accuracy': 0.7533103435136319, 'F1-Score': 0.528425428935922, 'AUC-PR': 0.5500542790039462, 'AUC-ROC': 0.8882511238338943, 'MCC': 0.5518174569625105, 'Specificity': 0.5218039508013418, 'Fit Status': 'Balanced Performance'}
Summary of All Model Results:
Best Parameters \
Decision Tree {'max_depth': None, 'min_samples_split': 2}
Random Forest {'max_depth': None, 'min_samples_split': 2, 'n...
XGBoost {'n_estimators': 50, 'max_depth': 5, 'learning...
LightGBM {'n_estimators': 50, 'max_depth': 10, 'learnin...
Gradient Boosting {'learning_rate': 0.1, 'max_depth': 5, 'n_esti...
Logistic Regression {'C': 1.0, 'penalty': 'l2'}
MLP {'activation': 'tanh', 'hidden_layer_sizes': (...
Train Accuracy Test Accuracy F1-Score AUC-PR \
Decision Tree 1.0 0.754377 0.501476 0.537704
Random Forest 1.0 0.81041 0.560589 0.582355
XGBoost 0.708603 0.740793 0.544559 0.595047
LightGBM 0.750986 0.754415 0.551507 0.592861
Gradient Boosting 0.771839 0.754187 0.549505 0.589644
Logistic Regression 0.671366 0.714005 0.518763 0.569872
MLP 0.827727 0.75331 0.528425 0.550054
AUC-ROC MCC Specificity \
Decision Tree 0.737595 0.535016 0.438315
Random Forest 0.910712 0.620309 0.506895
XGBoost 0.917692 0.547596 0.541558
LightGBM 0.917866 0.561307 0.55423
Gradient Boosting 0.915021 0.560343 0.55013
Logistic Regression 0.903881 0.514579 0.56243
MLP 0.888251 0.551817 0.521804
Fit Status
Decision Tree Potential Overfitting Detected
Random Forest Potential Overfitting Detected
XGBoost Balanced Performance
LightGBM Balanced Performance
Gradient Boosting Balanced Performance
Logistic Regression Balanced Performance
MLP Balanced Performance
In [128]:
import os
import joblib
def save_models_individually(model_dict, save_dir):
"""
Saves each model from the given dictionary to a .pkl file in the specified directory.
Filenames are derived from model names.
"""
os.makedirs(save_dir, exist_ok=True)
for model_name, model_obj in model_dict.items():
# Clean filename
clean_name = model_name.replace(" ", "_").replace("(", "").replace(")", "").replace("-", "")
filename = os.path.join(save_dir, f"{clean_name}.pkl")
joblib.dump(model_obj, filename)
print(f"Saved: {filename}")
save_models_individually(
model_dict=trained_models,
save_dir=r"C:\Users\ghaza\Desktop\FYP\Application\AI\models\Multi Training - GridSearch"
)
Saved: C:\Users\ghaza\Desktop\FYP\Application\AI\models\Multi Training - GridSearch\Decision_Tree.pkl Saved: C:\Users\ghaza\Desktop\FYP\Application\AI\models\Multi Training - GridSearch\Random_Forest.pkl Saved: C:\Users\ghaza\Desktop\FYP\Application\AI\models\Multi Training - GridSearch\XGBoost.pkl Saved: C:\Users\ghaza\Desktop\FYP\Application\AI\models\Multi Training - GridSearch\LightGBM.pkl Saved: C:\Users\ghaza\Desktop\FYP\Application\AI\models\Multi Training - GridSearch\Gradient_Boosting.pkl Saved: C:\Users\ghaza\Desktop\FYP\Application\AI\models\Multi Training - GridSearch\Logistic_Regression.pkl Saved: C:\Users\ghaza\Desktop\FYP\Application\AI\models\Multi Training - GridSearch\MLP.pkl
In [61]:
def plot_model_metric_bars(results_df):
"""
Plots grouped bar charts for each metric:
Train vs Test values side-by-side per model.
"""
# Define metrics (both train/test where applicable)
metric_groups = [
("Train Accuracy", "Test Accuracy", "Accuracy"),
("F1-Score", "F1-Score", "F1-Macro"),
("AUC-PR", "AUC-PR", "AUC-PR"),
("AUC-ROC", "AUC-ROC", "AUC-ROC")
]
color_train = "lightsteelblue"
color_test = "peachpuff"
for train_col, test_col, display_name in metric_groups:
if train_col not in results_df.columns or test_col not in results_df.columns:
print(f"Skipping {display_name}: missing columns.")
continue
x = np.arange(len(results_df))
width = 0.35
fig, ax = plt.subplots(figsize=(10, 6))
bars_train = ax.bar(x - width / 2, results_df[train_col], width, label="Train", color=color_train)
bars_test = ax.bar(x + width / 2, results_df[test_col], width, label="Test", color=color_test)
# Add data labels
for bars in [bars_train, bars_test]:
for bar in bars:
height = bar.get_height()
ax.text(
bar.get_x() + bar.get_width() / 2.0,
height,
f"{height:.2f}",
ha="center", va="bottom", fontsize=9
)
ax.set_ylabel(display_name)
ax.set_title(f"{display_name} Comparison (Train vs Test)")
ax.set_xticks(x)
ax.set_xticklabels(results_df.index, rotation=45, ha="right")
ax.legend()
plt.tight_layout()
plt.show()
In [62]:
plot_model_metric_bars(
results_df=summary_df # From your run_multiclass_grid_search
)
In [111]:
def plot_roc_pr_curves(model_dict, X_test, y_test, title_prefix="Model Evaluation"):
class_labels = np.unique(y_test)
y_test_bin = label_binarize(y_test, classes=class_labels)
for name, model in model_dict.items():
if hasattr(model, "predict_proba"):
try:
y_proba = model.predict_proba(X_test)
# === ROC Curve ===
plt.figure(figsize=(8, 5))
for i in range(len(class_labels)):
fpr, tpr, _ = roc_curve(y_test_bin[:, i], y_proba[:, i])
roc_auc = auc(fpr, tpr)
plt.plot(fpr, tpr, label=f"Class {class_labels[i]} (AUC = {roc_auc:.2f})")
plt.plot([0, 1], [0, 1], 'k--')
plt.title(f"{title_prefix} - ROC Curve: {name}")
plt.xlabel("False Positive Rate")
plt.ylabel("True Positive Rate")
plt.legend(loc="lower right", fontsize='small')
plt.grid(True)
plt.tight_layout()
plt.show()
# === Precision-Recall Curve ===
plt.figure(figsize=(8, 5))
for i in range(len(class_labels)):
precision, recall, _ = precision_recall_curve(y_test_bin[:, i], y_proba[:, i])
ap = average_precision_score(y_test_bin[:, i], y_proba[:, i])
plt.plot(recall, precision, label=f"Class {class_labels[i]} (AP = {ap:.2f})")
plt.title(f"{title_prefix} - Precision-Recall Curve: {name}")
plt.xlabel("Recall")
plt.ylabel("Precision")
plt.legend(loc="lower left", fontsize='small')
plt.grid(True)
plt.tight_layout()
plt.show()
except Exception as e:
print(f"Error plotting for {name}: {e}")
In [115]:
plot_roc_pr_curves(
model_dict=trained_models, # or all_models
X_test=X_test,
y_test=y_test_multi,
title_prefix="Initial Grid Search"
)
In [52]:
def explain_models_with_shap(models, X_train, X_test, sample_size=100, background_size=100):
import shap
import numpy as np
import pandas as pd
shap_values_main = {}
X_test_sample = X_test.sample(sample_size, random_state=42)
background = shap.sample(X_train, background_size, random_state=42)
for model_name, model in models.items():
print(f"\nSHAP for: {model_name}")
try:
# === Tree-based models with TreeExplainer ===
if model_name.lower().startswith(("xgboost", "lightgbm", "random forest")):
# Use model-specific booster when possible
explainer = shap.TreeExplainer(model)
shap_vals = explainer.shap_values(X_test_sample)
# === All others use KernelExplainer ===
else:
explainer = shap.KernelExplainer(model.predict_proba, background)
shap_vals = explainer.shap_values(X_test_sample)
# Handle multiclass
if isinstance(shap_vals, list):
shap_vals = shap_vals[1] if len(shap_vals) > 1 else shap_vals[0]
shap_vals_main = shap_vals[:, :, 1] if shap_vals.ndim == 3 else shap_vals
if shap_vals_main.shape[0] != X_test_sample.shape[0]:
print(f"Mismatch in SHAP shape for {model_name}")
continue
shap_values_main[model_name] = shap_vals_main
shap.summary_plot(shap_vals_main, X_test_sample, feature_names=X_train.columns, plot_type="dot", show=True)
except Exception as e:
print(f"SHAP failed for {model_name}: {e}")
return shap_values_main
def plot_mean_shap_summary(shap_values_main, feature_names):
shap_summary_df = pd.DataFrame({
model: np.abs(values).mean(axis=0)
for model, values in shap_values_main.items()
}, index=feature_names).T
mean_shap = shap_summary_df.mean(axis=0).sort_values(ascending=False)
plt.figure(figsize=(10, 6))
ax = sns.barplot(x=mean_shap.values, y=mean_shap.index, palette="Set3")
for rect in ax.patches:
width = rect.get_width()
ax.text(width + 0.001, rect.get_y() + rect.get_height() / 2, f"{width:.3f}", ha="left", va="center")
plt.title("Mean Absolute SHAP Value (Across All Models)")
plt.xlabel("SHAP Value")
plt.tight_layout()
plt.show()
In [54]:
# Explain and plot individual SHAP summary plots
shap_values_main = explain_models_with_shap(
models=trained_models,
X_train=X_train_multi,
X_test=X_test,
sample_size=100,
background_size=100
)
# Plot overall SHAP feature importance
plot_mean_shap_summary(
shap_values_main=shap_values_main,
feature_names=X_train_multi.columns
)
SHAP for: Decision Tree
0%| | 0/100 [00:00<?, ?it/s]
SHAP for: Random Forest
SHAP for: XGBoost
SHAP for: LightGBM
SHAP for: Gradient Boosting
0%| | 0/100 [00:00<?, ?it/s]
SHAP for: Logistic Regression
0%| | 0/100 [00:00<?, ?it/s]
SHAP for: MLP
0%| | 0/100 [00:00<?, ?it/s]
In [56]:
from sklearn.model_selection import learning_curve
import pandas as pd
def get_learning_curve_data(estimator, X, y, cv, scoring="f1_macro", n_jobs=-1, train_sizes=np.linspace(0.1, 1.0, 5)):
"""
Returns a DataFrame with training size, train and CV scores (mean and std) for a given model.
"""
train_sizes, train_scores, cv_scores = learning_curve(
estimator,
X,
y,
cv=cv,
scoring=scoring,
n_jobs=n_jobs,
train_sizes=train_sizes,
shuffle=True,
random_state=42
)
df = pd.DataFrame({
"Training Size": train_sizes,
"Train Score Mean": train_scores.mean(axis=1),
"Train Score Std": train_scores.std(axis=1),
"CV Score Mean": cv_scores.mean(axis=1),
"CV Score Std": cv_scores.std(axis=1)
})
return df
In [58]:
# Create the dictionary
learning_curve_data_dict = {}
# You can use your cv_strategy from before (e.g., StratifiedKFold)
for model_name, model in trained_models.items():
print(f"Generating learning curve for: {model_name}")
try:
df_lc = get_learning_curve_data(
estimator=model,
X=X_train_multi,
y=y_train_multi,
cv=cv_strategy,
scoring="f1_macro"
)
learning_curve_data_dict[model_name] = df_lc
except Exception as e:
print(f"Failed for {model_name}: {e}")
Generating learning curve for: Decision Tree Generating learning curve for: Random Forest Generating learning curve for: XGBoost Failed for XGBoost: 'super' object has no attribute '__sklearn_tags__' Generating learning curve for: LightGBM Generating learning curve for: Gradient Boosting Generating learning curve for: Logistic Regression Generating learning curve for: MLP
In [59]:
def plot_combined_learning_curves(learning_curve_data_dict, title_prefix="Learning Curve"):
for model_name, df in learning_curve_data_dict.items():
if not isinstance(df, pd.DataFrame):
continue
x = df["Training Size"].values
plt.figure(figsize=(10, 5))
plt.plot(x, df["Train Score Mean"], 'o-', label="Train", color='tomato')
plt.fill_between(x, df["Train Score Mean"] - df["Train Score Std"], df["Train Score Mean"] + df["Train Score Std"], alpha=0.1, color='tomato')
plt.plot(x, df["CV Score Mean"], 's--', label="CV", color='seagreen')
plt.fill_between(x, df["CV Score Mean"] - df["CV Score Std"], df["CV Score Mean"] + df["CV Score Std"], alpha=0.1, color='seagreen')
plt.title(f"{title_prefix} - {model_name}")
plt.xlabel("Training Examples")
plt.ylabel("F1 Score")
plt.legend()
plt.grid(True)
plt.tight_layout()
plt.show()
In [60]:
plot_combined_learning_curves(
learning_curve_data_dict=learning_curve_data_dict,
title_prefix="Initial Grid Search"
)
In [67]:
regularised_models = {
"Decision Tree": DecisionTreeClassifier(max_depth=5, min_samples_split=10, ccp_alpha=0.01, random_state=42),
"Random Forest": RandomForestClassifier(n_estimators=100, max_depth=8, min_samples_leaf=5, min_samples_split=10, random_state=42),
"XGBoost": XGBClassifier(n_estimators=50, max_depth=4, learning_rate=0.05, subsample=0.8, colsample_bytree=0.8, objective="multi:softprob", num_class=5, use_label_encoder=False, eval_metric="mlogloss", verbosity=0, random_state=42),
"LightGBM": LGBMClassifier(n_estimators=50, max_depth=6, learning_rate=0.05, subsample=0.8, colsample_bytree=0.8, objective="multiclass", num_class=5, random_state=42),
"Gradient Boosting": GradientBoostingClassifier(n_estimators=100, learning_rate=0.05, max_depth=4, subsample=0.8, random_state=42),
"Logistic Regression": LogisticRegression(C=0.05, penalty="l2", multi_class="multinomial", solver="lbfgs", max_iter=1000, random_state=42),
"MLP": MLPClassifier(hidden_layer_sizes=(50, 50), activation="tanh", alpha=0.01, learning_rate_init=0.001, max_iter=300, early_stopping=True, random_state=42)
}
fine_tune_grids = {
"Decision Tree": {
"max_depth": [None, 10, 20],
"min_samples_split": [2, 5, 10],
},
"Random Forest": {
"n_estimators": [100, 200],
"max_depth": [None, 10],
"min_samples_split": [2, 5],
},
"XGBoost": {
"n_estimators": [50, 100],
"max_depth": [3, 5, 7],
"learning_rate": [0.05, 0.1],
},
"LightGBM": {
"n_estimators": [50, 100],
"max_depth": [5, 10],
"learning_rate": [0.05, 0.1],
},
"Gradient Boosting": {
"n_estimators": [100, 200],
"max_depth": [3, 5],
"learning_rate": [0.05, 0.1],
},
"Logistic Regression": {
"C": [0.1, 1.0, 10],
"penalty": ["l2"]
},
"MLP": {
"hidden_layer_sizes": [(50,), (50, 50)],
"activation": ["relu", "tanh"],
"learning_rate_init": [0.001, 0.01]
}
}
In [87]:
# Drop Coolant_Temperature from training and test sets
X_train_multi_dropped = X_train_multi.drop(columns=["Coolant_Temperature"])
X_test_dropped = X_test.drop(columns=["Coolant_Temperature"])
In [93]:
# Save the dropped feature datasets to CSV
X_train_multi_dropped.to_csv(r"C:\Users\ghaza\Desktop\FYP\Application\AI\data\dropped_final_training\X_train_multi_dropped.csv", index=False)
X_test_dropped.to_csv(r"C:\Users\ghaza\Desktop\FYP\Application\AI\data\dropped_final_training\X_test_multi_dropped.csv", index=False)
print("Files saved successfully.")
Files saved successfully.
In [150]:
from itertools import product
import pandas as pd
import numpy as np
from sklearn.base import clone, is_classifier
from sklearn.model_selection import StratifiedKFold, cross_val_score
def safe_cross_val_score(estimator, X, y, cv, scoring):
try:
if not is_classifier(estimator):
return np.array([np.nan])
return cross_val_score(estimator, X, y, cv=cv, scoring=scoring, n_jobs=-1)
except Exception as e:
print(f"Cross-validation failed: {e}")
return np.array([np.nan])
In [162]:
def evaluate_pipeline_multiclass_with_manual_tuning(
regularised_models, fine_tune_grids, X_train, y_train, X_test, y_test, cv_folds=5
):
cv = StratifiedKFold(n_splits=cv_folds, shuffle=True, random_state=42)
all_models = {}
reg_results, ft_results = {}, {}
cm_dict_reg, cm_dict_ft = {}, {}
def print_summary(name, model, X_train, y_train, X_test, y_test, best_params, fit_status, auc_pr, auc_roc, mcc, specificity):
y_pred = model.predict(X_test)
print(f"\nStarting grid search for {name}...")
print(classification_report(y_test, y_pred))
print(f"\nResults for {name}:")
results = {
"Best Parameters": best_params,
"Train Accuracy": accuracy_score(y_train, model.predict(X_train)),
"Test Accuracy": accuracy_score(y_test, y_pred),
"F1-Score": f1_score(y_test, y_pred, average="macro"),
"AUC-PR": auc_pr,
"AUC-ROC": auc_roc,
"MCC": mcc,
"Specificity": specificity,
"Fit Status": fit_status
}
for k, v in results.items():
print(f"{k}: {v}")
print("-" * 80)
for model_name, base_model in regularised_models.items():
print(f"\nProcessing {model_name}...")
# === REGULARISED ===
reg_model = clone(base_model)
reg_model.fit(X_train, y_train)
cv_scores = safe_cross_val_score(reg_model, X_train, y_train, cv=cv, scoring="f1_macro")
y_train_pred = reg_model.predict(X_train)
y_test_pred = reg_model.predict(X_test)
train_acc = accuracy_score(y_train, y_train_pred)
test_acc = accuracy_score(y_test, y_test_pred)
fit_status = (
"Potential Overfitting Detected!" if train_acc > test_acc + 0.1 else
"Potential Underfitting Detected!" if test_acc > train_acc + 0.1 else
"Balanced Performance"
)
auc_roc = auc_pr = None
if hasattr(reg_model, "predict_proba"):
try:
y_probs = reg_model.predict_proba(X_test)
auc_roc = roc_auc_score(y_test, y_probs, multi_class="ovr")
auc_pr = np.mean([
auc(*precision_recall_curve((y_test == i).astype(int), y_probs[:, i])[1::-1])
for i in np.unique(y_test)
])
except Exception:
pass
cm = confusion_matrix(y_test, y_test_pred)
specificity = cm[0, 0] / np.sum(cm[0]) if cm.shape[0] > 1 else None
cm_dict_reg[model_name] = cm
reg_results[model_name] = {
"Train Accuracy": train_acc,
"Test Accuracy": test_acc,
"F1-Macro": f1_score(y_test, y_test_pred, average="macro"),
"Precision": precision_score(y_test, y_test_pred, average="macro"),
"Recall": recall_score(y_test, y_test_pred, average="macro"),
"MCC": matthews_corrcoef(y_test, y_test_pred),
"AUC-PR": auc_pr,
"AUC-ROC": auc_roc,
"Specificity": specificity,
"CV F1 Mean": np.nanmean(cv_scores),
"CV F1 Std": np.nanstd(cv_scores),
"Fit Status": fit_status
}
all_models[f"{model_name} - Regularised"] = reg_model
print_summary(
name=model_name + " (Regularised)",
model=reg_model,
X_train=X_train,
y_train=y_train,
X_test=X_test,
y_test=y_test,
best_params=reg_model.get_params(),
fit_status=fit_status,
auc_pr=auc_pr,
auc_roc=auc_roc,
mcc=matthews_corrcoef(y_test, y_test_pred),
specificity=specificity
)
# === CONDITIONAL TUNING ===
if model_name in ["XGBoost", "Gradient Boosting", "LightGBM"]:
print(f"Manually tuning {model_name}...")
best_score = -np.inf
best_model = None
best_params = None
for combo in product(*fine_tune_grids[model_name].values()):
params = dict(zip(fine_tune_grids[model_name].keys(), combo))
try:
tuned_model = clone(base_model).set_params(**params)
tuned_model.fit(X_train, y_train)
preds = tuned_model.predict(X_test)
score = f1_score(y_test, preds, average="macro")
if score > best_score:
best_score = score
best_model = tuned_model
best_params = params
except Exception as e:
print(f"Skipping {params} due to error: {e}")
else:
print(f"Fine-tuning {model_name} using HalvingGridSearchCV...")
halving_grid = HalvingGridSearchCV(
estimator=clone(base_model),
param_grid=fine_tune_grids[model_name],
scoring="f1_macro",
cv=cv,
n_jobs=-1,
factor=2,
verbose=0
)
halving_grid.fit(X_train, y_train)
best_model = halving_grid.best_estimator_
best_params = halving_grid.best_params_
y_train_ft = best_model.predict(X_train)
y_test_ft = best_model.predict(X_test)
train_acc_ft = accuracy_score(y_train, y_train_ft)
test_acc_ft = accuracy_score(y_test, y_test_ft)
fit_status_ft = (
"Potential Overfitting Detected!" if train_acc_ft > test_acc_ft + 0.1 else
"Potential Underfitting Detected!" if test_acc_ft > train_acc_ft + 0.1 else
"Balanced Performance"
)
auc_roc_ft = auc_pr_ft = None
if hasattr(best_model, "predict_proba"):
try:
y_probs_ft = best_model.predict_proba(X_test)
auc_roc_ft = roc_auc_score(y_test, y_probs_ft, multi_class="ovr")
auc_pr_ft = np.mean([
auc(*precision_recall_curve((y_test == i).astype(int), y_probs_ft[:, i])[1::-1])
for i in np.unique(y_test)
])
except Exception:
pass
cm_ft = confusion_matrix(y_test, y_test_ft)
specificity_ft = cm_ft[0, 0] / cm_ft[0].sum() if cm_ft.shape[0] > 1 else None
cm_dict_ft[model_name] = cm_ft
ft_results[model_name] = {
"Best Params": best_params,
"Train Accuracy": train_acc_ft,
"Test Accuracy": test_acc_ft,
"F1-Macro": f1_score(y_test, y_test_ft, average="macro"),
"Precision": precision_score(y_test, y_test_ft, average="macro"),
"Recall": recall_score(y_test, y_test_ft, average="macro"),
"MCC": matthews_corrcoef(y_test, y_test_ft),
"AUC-PR": auc_pr_ft,
"AUC-ROC": auc_roc_ft,
"CV F1 Mean": np.nan,
"CV F1 Std": np.nan,
"Fit Status": fit_status_ft
}
all_models[f"{model_name} - FineTuned"] = best_model
print_summary(
name=model_name + " (Fine-Tuned)",
model=best_model,
X_train=X_train,
y_train=y_train,
X_test=X_test,
y_test=y_test,
best_params=best_params,
fit_status=fit_status_ft,
auc_pr=auc_pr_ft,
auc_roc=auc_roc_ft,
mcc=matthews_corrcoef(y_test, y_test_ft),
specificity=specificity_ft
)
df_reg = pd.DataFrame(reg_results).T
df_ft = pd.DataFrame(ft_results).T
print("\nSummary - Regularised Models:\n", df_reg)
print("\nSummary - Fine-Tuned Models:\n", df_ft)
return all_models, reg_results, ft_results, df_reg, df_ft, cm_dict_reg, cm_dict_ft
In [164]:
all_models, reg_results, ft_results, df_reg, df_ft, cm_dict_reg, cm_dict_ft = evaluate_pipeline_multiclass_with_manual_tuning(
regularised_models=regularised_models,
fine_tune_grids=fine_tune_grids,
X_train=X_train_multi_dropped,
y_train=y_train_multi,
X_test=X_test_dropped,
y_test=y_test_multi
)
Processing Decision Tree...
Starting grid search for Decision Tree (Regularised)...
precision recall f1-score support
0 0.41 0.61 0.49 2683
1 0.94 0.66 0.78 38015
2 0.04 0.64 0.07 797
3 0.64 0.70 0.67 5512
4 0.50 0.19 0.27 5480
accuracy 0.61 52487
macro avg 0.51 0.56 0.46 52487
weighted avg 0.83 0.61 0.69 52487
Results for Decision Tree (Regularised):
Best Parameters: {'ccp_alpha': 0.01, 'class_weight': None, 'criterion': 'gini', 'max_depth': 5, 'max_features': None, 'max_leaf_nodes': None, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 1, 'min_samples_split': 10, 'min_weight_fraction_leaf': 0.0, 'monotonic_cst': None, 'random_state': 42, 'splitter': 'best'}
Train Accuracy: 0.6192233691114365
Test Accuracy: 0.6116181149618001
F1-Score: 0.4564529763100015
AUC-PR: 0.5704715342960476
AUC-ROC: 0.8300965613486367
MCC: 0.4084862600275739
Specificity: 0.6067834513604174
Fit Status: Balanced Performance
--------------------------------------------------------------------------------
Fine-tuning Decision Tree using HalvingGridSearchCV...
Starting grid search for Decision Tree (Fine-Tuned)...
precision recall f1-score support
0 0.41 0.61 0.49 2683
1 0.94 0.66 0.78 38015
2 0.04 0.64 0.07 797
3 0.64 0.70 0.67 5512
4 0.50 0.19 0.27 5480
accuracy 0.61 52487
macro avg 0.51 0.56 0.46 52487
weighted avg 0.83 0.61 0.69 52487
Results for Decision Tree (Fine-Tuned):
Best Parameters: {'max_depth': 20, 'min_samples_split': 5}
Train Accuracy: 0.6192233691114365
Test Accuracy: 0.6116181149618001
F1-Score: 0.4564529763100015
AUC-PR: 0.5704715342960476
AUC-ROC: 0.8300965613486367
MCC: 0.4084862600275739
Specificity: 0.6067834513604174
Fit Status: Balanced Performance
--------------------------------------------------------------------------------
Processing Random Forest...
Starting grid search for Random Forest (Regularised)...
precision recall f1-score support
0 0.49 0.57 0.53 2683
1 0.96 0.77 0.85 38015
2 0.05 0.61 0.09 797
3 0.72 0.85 0.78 5512
4 0.52 0.22 0.31 5480
accuracy 0.71 52487
macro avg 0.55 0.60 0.51 52487
weighted avg 0.85 0.71 0.76 52487
Results for Random Forest (Regularised):
Best Parameters: {'bootstrap': True, 'ccp_alpha': 0.0, 'class_weight': None, 'criterion': 'gini', 'max_depth': 8, 'max_features': 'sqrt', 'max_leaf_nodes': None, 'max_samples': None, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 5, 'min_samples_split': 10, 'min_weight_fraction_leaf': 0.0, 'monotonic_cst': None, 'n_estimators': 100, 'n_jobs': None, 'oob_score': False, 'random_state': 42, 'verbose': 0, 'warm_start': False}
Train Accuracy: 0.6853664734568424
Test Accuracy: 0.7064987520719416
F1-Score: 0.5117880154629761
AUC-PR: 0.5763380024301908
AUC-ROC: 0.909100673499273
MCC: 0.5060311414345766
Specificity: 0.5736116287737607
Fit Status: Balanced Performance
--------------------------------------------------------------------------------
Fine-tuning Random Forest using HalvingGridSearchCV...
Starting grid search for Random Forest (Fine-Tuned)...
precision recall f1-score support
0 0.45 0.52 0.48 2683
1 0.96 0.84 0.89 38015
2 0.05 0.14 0.07 797
3 0.73 0.85 0.79 5512
4 0.40 0.54 0.46 5480
accuracy 0.78 52487
macro avg 0.52 0.58 0.54 52487
weighted avg 0.83 0.78 0.80 52487
Results for Random Forest (Fine-Tuned):
Best Parameters: {'max_depth': None, 'min_samples_split': 5, 'n_estimators': 200}
Train Accuracy: 0.9818033858105737
Test Accuracy: 0.7814315925848305
F1-Score: 0.538167899107699
AUC-PR: 0.5670788669572466
AUC-ROC: 0.9069611125710955
MCC: 0.5786120599903005
Specificity: 0.5184494968319046
Fit Status: Potential Overfitting Detected!
--------------------------------------------------------------------------------
Processing XGBoost...
Cross-validation failed: 'super' object has no attribute '__sklearn_tags__'
Starting grid search for XGBoost (Regularised)...
precision recall f1-score support
0 0.50 0.55 0.52 2683
1 0.96 0.79 0.87 38015
2 0.05 0.59 0.09 797
3 0.71 0.85 0.78 5512
4 0.51 0.25 0.33 5480
accuracy 0.72 52487
macro avg 0.55 0.60 0.52 52487
weighted avg 0.85 0.72 0.77 52487
Results for XGBoost (Regularised):
Best Parameters: {'objective': 'multi:softprob', 'base_score': None, 'booster': None, 'callbacks': None, 'colsample_bylevel': None, 'colsample_bynode': None, 'colsample_bytree': 0.8, 'device': None, 'early_stopping_rounds': None, 'enable_categorical': False, 'eval_metric': 'mlogloss', 'feature_types': None, 'gamma': None, 'grow_policy': None, 'importance_type': None, 'interaction_constraints': None, 'learning_rate': 0.05, 'max_bin': None, 'max_cat_threshold': None, 'max_cat_to_onehot': None, 'max_delta_step': None, 'max_depth': 4, 'max_leaves': None, 'min_child_weight': None, 'missing': nan, 'monotone_constraints': None, 'multi_strategy': None, 'n_estimators': 50, 'n_jobs': None, 'num_parallel_tree': None, 'random_state': 42, 'reg_alpha': None, 'reg_lambda': None, 'sampling_method': None, 'scale_pos_weight': None, 'subsample': 0.8, 'tree_method': None, 'validate_parameters': None, 'verbosity': 0, 'num_class': 5, 'use_label_encoder': False}
Train Accuracy: 0.6779879959825265
Test Accuracy: 0.7227885000095262
F1-Score: 0.5180339947021662
AUC-PR: 0.5787849701545823
AUC-ROC: 0.9093553276540376
MCC: 0.5188568430201828
Specificity: 0.5501304509877003
Fit Status: Balanced Performance
--------------------------------------------------------------------------------
Manually tuning XGBoost...
Starting grid search for XGBoost (Fine-Tuned)...
precision recall f1-score support
0 0.49 0.54 0.51 2683
1 0.96 0.81 0.88 38015
2 0.05 0.42 0.09 797
3 0.72 0.86 0.78 5512
4 0.47 0.37 0.41 5480
accuracy 0.75 52487
macro avg 0.54 0.60 0.53 52487
weighted avg 0.85 0.75 0.79 52487
Results for XGBoost (Fine-Tuned):
Best Parameters: {'n_estimators': 100, 'max_depth': 7, 'learning_rate': 0.1}
Train Accuracy: 0.7494524377110081
Test Accuracy: 0.7455560424486063
F1-Score: 0.5346915593883044
AUC-PR: 0.5812320297583501
AUC-ROC: 0.9129681551568705
MCC: 0.5426292518560163
Specificity: 0.5363399180022363
Fit Status: Balanced Performance
--------------------------------------------------------------------------------
Processing LightGBM...
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.010752 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1530
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 6
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
Starting grid search for LightGBM (Regularised)...
precision recall f1-score support
0 0.51 0.54 0.52 2683
1 0.96 0.79 0.87 38015
2 0.05 0.55 0.09 797
3 0.72 0.85 0.78 5512
4 0.50 0.29 0.37 5480
accuracy 0.73 52487
macro avg 0.55 0.61 0.53 52487
weighted avg 0.85 0.73 0.78 52487
Results for LightGBM (Regularised):
Best Parameters: {'boosting_type': 'gbdt', 'class_weight': None, 'colsample_bytree': 0.8, 'importance_type': 'split', 'learning_rate': 0.05, 'max_depth': 6, 'min_child_samples': 20, 'min_child_weight': 0.001, 'min_split_gain': 0.0, 'n_estimators': 50, 'n_jobs': None, 'num_leaves': 31, 'objective': 'multiclass', 'random_state': 42, 'reg_alpha': 0.0, 'reg_lambda': 0.0, 'subsample': 0.8, 'subsample_for_bin': 200000, 'subsample_freq': 0, 'num_class': 5}
Train Accuracy: 0.700788973729111
Test Accuracy: 0.7261036066073504
F1-Score: 0.5264928694119189
AUC-PR: 0.5816104379567691
AUC-ROC: 0.9126329537478147
MCC: 0.5254275158890521
Specificity: 0.5400670890793887
Fit Status: Balanced Performance
--------------------------------------------------------------------------------
Manually tuning LightGBM...
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.009195 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1530
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 6
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.011043 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1530
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 6
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.009226 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1530
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 6
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.009780 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1530
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 6
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.011821 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1530
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 6
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.010522 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1530
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 6
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.009664 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1530
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 6
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.010194 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1530
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 6
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
Starting grid search for LightGBM (Fine-Tuned)...
precision recall f1-score support
0 0.50 0.54 0.52 2683
1 0.96 0.79 0.87 38015
2 0.05 0.50 0.09 797
3 0.73 0.85 0.79 5512
4 0.50 0.34 0.40 5480
accuracy 0.73 52487
macro avg 0.55 0.60 0.53 52487
weighted avg 0.85 0.73 0.78 52487
Results for LightGBM (Fine-Tuned):
Best Parameters: {'n_estimators': 50, 'max_depth': 5, 'learning_rate': 0.1}
Train Accuracy: 0.7058955214850131
Test Accuracy: 0.734581896469602
F1-Score: 0.5338048635612594
AUC-PR: 0.5828284215169861
AUC-ROC: 0.9138955036568385
MCC: 0.5340369506312618
Specificity: 0.5408125232948192
Fit Status: Balanced Performance
--------------------------------------------------------------------------------
Processing Gradient Boosting...
Starting grid search for Gradient Boosting (Regularised)...
precision recall f1-score support
0 0.50 0.54 0.52 2683
1 0.96 0.78 0.86 38015
2 0.05 0.48 0.08 797
3 0.73 0.85 0.78 5512
4 0.48 0.35 0.41 5480
accuracy 0.72 52487
macro avg 0.54 0.60 0.53 52487
weighted avg 0.85 0.72 0.78 52487
Results for Gradient Boosting (Regularised):
Best Parameters: {'ccp_alpha': 0.0, 'criterion': 'friedman_mse', 'init': None, 'learning_rate': 0.05, 'loss': 'log_loss', 'max_depth': 4, 'max_features': None, 'max_leaf_nodes': None, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 1, 'min_samples_split': 2, 'min_weight_fraction_leaf': 0.0, 'n_estimators': 100, 'n_iter_no_change': None, 'random_state': 42, 'subsample': 0.8, 'tol': 0.0001, 'validation_fraction': 0.1, 'verbose': 0, 'warm_start': False}
Train Accuracy: 0.7070723266254432
Test Accuracy: 0.7235124888067521
F1-Score: 0.5314326045798792
AUC-PR: 0.5801242007980838
AUC-ROC: 0.9107400069225646
MCC: 0.5250754674453689
Specificity: 0.5445396943719717
Fit Status: Balanced Performance
--------------------------------------------------------------------------------
Manually tuning Gradient Boosting...
Starting grid search for Gradient Boosting (Fine-Tuned)...
precision recall f1-score support
0 0.49 0.55 0.52 2683
1 0.96 0.79 0.87 38015
2 0.05 0.40 0.08 797
3 0.72 0.86 0.78 5512
4 0.45 0.39 0.42 5480
accuracy 0.74 52487
macro avg 0.53 0.60 0.54 52487
weighted avg 0.85 0.74 0.78 52487
Results for Gradient Boosting (Fine-Tuned):
Best Parameters: {'n_estimators': 200, 'max_depth': 3, 'learning_rate': 0.1}
Train Accuracy: 0.7233691114364889
Test Accuracy: 0.7393259283251091
F1-Score: 0.5351175033518946
AUC-PR: 0.5803869487166038
AUC-ROC: 0.911762257502688
MCC: 0.538531361066165
Specificity: 0.5516213194185613
Fit Status: Balanced Performance
--------------------------------------------------------------------------------
Processing Logistic Regression...
Starting grid search for Logistic Regression (Regularised)...
precision recall f1-score support
0 0.44 0.56 0.50 2683
1 0.96 0.76 0.85 38015
2 0.05 0.52 0.09 797
3 0.67 0.83 0.74 5512
4 0.43 0.28 0.34 5480
accuracy 0.70 52487
macro avg 0.51 0.59 0.50 52487
weighted avg 0.83 0.70 0.75 52487
Results for Logistic Regression (Regularised):
Best Parameters: {'C': 0.05, 'class_weight': None, 'dual': False, 'fit_intercept': True, 'intercept_scaling': 1, 'l1_ratio': None, 'max_iter': 1000, 'multi_class': 'multinomial', 'n_jobs': None, 'penalty': 'l2', 'random_state': 42, 'solver': 'lbfgs', 'tol': 0.0001, 'verbose': 0, 'warm_start': False}
Train Accuracy: 0.657849199530488
Test Accuracy: 0.7021738716253548
F1-Score: 0.5022068257562184
AUC-PR: 0.5547073323080622
AUC-ROC: 0.8983600298483101
MCC: 0.49541005283530465
Specificity: 0.5628028326500186
Fit Status: Balanced Performance
--------------------------------------------------------------------------------
Fine-tuning Logistic Regression using HalvingGridSearchCV...
Starting grid search for Logistic Regression (Fine-Tuned)...
precision recall f1-score support
0 0.44 0.56 0.50 2683
1 0.96 0.76 0.85 38015
2 0.05 0.52 0.09 797
3 0.67 0.83 0.74 5512
4 0.43 0.28 0.34 5480
accuracy 0.70 52487
macro avg 0.51 0.59 0.50 52487
weighted avg 0.83 0.70 0.75 52487
Results for Logistic Regression (Fine-Tuned):
Best Parameters: {'C': 0.1, 'penalty': 'l2'}
Train Accuracy: 0.6578522247365045
Test Accuracy: 0.7020786099415093
F1-Score: 0.5021753711489946
AUC-PR: 0.5546720448568512
AUC-ROC: 0.8983543789694515
MCC: 0.49532214540077857
Specificity: 0.5628028326500186
Fit Status: Balanced Performance
--------------------------------------------------------------------------------
Processing MLP...
Starting grid search for MLP (Regularised)...
precision recall f1-score support
0 0.41 0.53 0.46 2683
1 0.96 0.79 0.87 38015
2 0.05 0.29 0.08 797
3 0.70 0.85 0.77 5512
4 0.37 0.43 0.40 5480
accuracy 0.74 52487
macro avg 0.50 0.58 0.52 52487
weighted avg 0.83 0.74 0.78 52487
Results for MLP (Regularised):
Best Parameters: {'activation': 'tanh', 'alpha': 0.01, 'batch_size': 'auto', 'beta_1': 0.9, 'beta_2': 0.999, 'early_stopping': True, 'epsilon': 1e-08, 'hidden_layer_sizes': (50, 50), 'learning_rate': 'constant', 'learning_rate_init': 0.001, 'max_fun': 15000, 'max_iter': 300, 'momentum': 0.9, 'n_iter_no_change': 10, 'nesterovs_momentum': True, 'power_t': 0.5, 'random_state': 42, 'shuffle': True, 'solver': 'adam', 'tol': 0.0001, 'validation_fraction': 0.1, 'verbose': False, 'warm_start': False}
Train Accuracy: 0.7733424896235433
Test Accuracy: 0.7369253338922018
F1-Score: 0.5162021263461105
AUC-PR: 0.5452388092679551
AUC-ROC: 0.8938913842139298
MCC: 0.5296972751558054
Specificity: 0.5285128587402161
Fit Status: Balanced Performance
--------------------------------------------------------------------------------
Fine-tuning MLP using HalvingGridSearchCV...
Starting grid search for MLP (Fine-Tuned)...
precision recall f1-score support
0 0.41 0.51 0.46 2683
1 0.96 0.77 0.86 38015
2 0.04 0.34 0.07 797
3 0.76 0.82 0.79 5512
4 0.38 0.45 0.41 5480
accuracy 0.72 52487
macro avg 0.51 0.58 0.52 52487
weighted avg 0.84 0.72 0.77 52487
Results for MLP (Fine-Tuned):
Best Parameters: {'activation': 'tanh', 'hidden_layer_sizes': (50, 50), 'learning_rate_init': 0.01}
Train Accuracy: 0.7600104067086969
Test Accuracy: 0.7235124888067521
F1-Score: 0.5179979637492016
AUC-PR: 0.5462513604457632
AUC-ROC: 0.8901735492278563
MCC: 0.5178724866426806
Specificity: 0.5124860231084607
Fit Status: Balanced Performance
--------------------------------------------------------------------------------
Summary - Regularised Models:
Train Accuracy Test Accuracy F1-Macro Precision \
Decision Tree 0.619223 0.611618 0.456453 0.507176
Random Forest 0.685366 0.706499 0.511788 0.547421
XGBoost 0.677988 0.722789 0.518034 0.546501
LightGBM 0.700789 0.726104 0.526493 0.54923
Gradient Boosting 0.707072 0.723512 0.531433 0.543546
Logistic Regression 0.657849 0.702174 0.502207 0.510532
MLP 0.773342 0.736925 0.516202 0.49961
Recall MCC AUC-PR AUC-ROC Specificity \
Decision Tree 0.56017 0.408486 0.570472 0.830097 0.606783
Random Forest 0.603762 0.506031 0.576338 0.909101 0.573612
XGBoost 0.604678 0.518857 0.578785 0.909355 0.55013
LightGBM 0.605305 0.525428 0.58161 0.912633 0.540067
Gradient Boosting 0.601762 0.525075 0.580124 0.91074 0.54454
Logistic Regression 0.590926 0.49541 0.554707 0.89836 0.562803
MLP 0.578315 0.529697 0.545239 0.893891 0.528513
CV F1 Mean CV F1 Std Fit Status
Decision Tree 0.613008 0.001131 Balanced Performance
Random Forest 0.673387 0.001453 Balanced Performance
XGBoost NaN NaN Balanced Performance
LightGBM 0.692176 0.00124 Balanced Performance
Gradient Boosting 0.701366 0.001139 Balanced Performance
Logistic Regression 0.651506 0.001723 Balanced Performance
MLP 0.769928 0.003396 Balanced Performance
Summary - Fine-Tuned Models:
Best Params \
Decision Tree {'max_depth': 20, 'min_samples_split': 5}
Random Forest {'max_depth': None, 'min_samples_split': 5, 'n...
XGBoost {'n_estimators': 100, 'max_depth': 7, 'learnin...
LightGBM {'n_estimators': 50, 'max_depth': 5, 'learning...
Gradient Boosting {'n_estimators': 200, 'max_depth': 3, 'learnin...
Logistic Regression {'C': 0.1, 'penalty': 'l2'}
MLP {'activation': 'tanh', 'hidden_layer_sizes': (...
Train Accuracy Test Accuracy F1-Macro Precision \
Decision Tree 0.619223 0.611618 0.456453 0.507176
Random Forest 0.981803 0.781432 0.538168 0.517211
XGBoost 0.749452 0.745556 0.534692 0.538352
LightGBM 0.705896 0.734582 0.533805 0.54871
Gradient Boosting 0.723369 0.739326 0.535118 0.53493
Logistic Regression 0.657852 0.702079 0.502175 0.510491
MLP 0.76001 0.723512 0.517998 0.512388
Recall MCC AUC-PR AUC-ROC CV F1 Mean \
Decision Tree 0.56017 0.408486 0.570472 0.830097 NaN
Random Forest 0.575766 0.578612 0.567079 0.906961 NaN
XGBoost 0.597378 0.542629 0.581232 0.912968 NaN
LightGBM 0.604137 0.534037 0.582828 0.913896 NaN
Gradient Boosting 0.599081 0.538531 0.580387 0.911762 NaN
Logistic Regression 0.590899 0.495322 0.554672 0.898354 NaN
MLP 0.576717 0.517872 0.546251 0.890174 NaN
CV F1 Std Fit Status
Decision Tree NaN Balanced Performance
Random Forest NaN Potential Overfitting Detected!
XGBoost NaN Balanced Performance
LightGBM NaN Balanced Performance
Gradient Boosting NaN Balanced Performance
Logistic Regression NaN Balanced Performance
MLP NaN Balanced Performance
In [187]:
import os
import joblib
# Save path
export_path = r"C:\Users\ghaza\Desktop\FYP\Application\AI\models\Finetuned_Reg_Multiclass"
os.makedirs(export_path, exist_ok=True)
# Save models with clean filenames
for model_key, model in all_models.items():
# Clean filename: lowercase, replace spaces with underscores
filename = model_key.lower().replace(" ", "_").replace("-", "").replace("__", "_") + ".pkl"
filepath = os.path.join(export_path, filename)
joblib.dump(model, filepath)
print(f"Exported {len(all_models)} models to: {export_path}")
Exported 14 models to: C:\Users\ghaza\Desktop\FYP\Application\AI\models\Finetuned_Reg_Multiclass
In [167]:
df_reg.index = [f"{name} (Reg)" for name in df_reg.index]
df_ft.index = [f"{name} (Tuned)" for name in df_ft.index]
combined_df = pd.concat([df_reg, df_ft])
In [182]:
plot_model_metric_bars(combined_df)
Skipping F1-Macro: missing columns.
In [169]:
plot_roc_pr_curves(
model_dict=all_models,
X_test=X_test_dropped,
y_test=y_test_multi,
title_prefix="Regularlised vs Halving Grid Search"
)
In [170]:
shap_values_main = explain_models_with_shap(
models=all_models,
X_train=X_train_multi_dropped,
X_test=X_test_dropped,
sample_size=100,
background_size=100
)
plot_mean_shap_summary(
shap_values_main=shap_values_main,
feature_names=X_train_multi_dropped.columns
)
SHAP for: Decision Tree - Regularised
0%| | 0/100 [00:00<?, ?it/s]
SHAP for: Decision Tree - FineTuned
0%| | 0/100 [00:00<?, ?it/s]
SHAP for: Random Forest - Regularised
SHAP for: Random Forest - FineTuned
SHAP for: XGBoost - Regularised
SHAP for: XGBoost - FineTuned
SHAP for: LightGBM - Regularised
SHAP for: LightGBM - FineTuned
SHAP for: Gradient Boosting - Regularised
0%| | 0/100 [00:00<?, ?it/s]
SHAP for: Gradient Boosting - FineTuned
0%| | 0/100 [00:00<?, ?it/s]
SHAP for: Logistic Regression - Regularised
0%| | 0/100 [00:00<?, ?it/s]
SHAP for: Logistic Regression - FineTuned
0%| | 0/100 [00:00<?, ?it/s]
SHAP for: MLP - Regularised
0%| | 0/100 [00:00<?, ?it/s]
SHAP for: MLP - FineTuned
0%| | 0/100 [00:00<?, ?it/s]
In [165]:
learning_curve_data_dict = {}
for model_name, model in all_models.items():
print(f"Generating learning curve for: {model_name}")
try:
df_lc = get_learning_curve_data(
estimator=model,
X=X_train_multi_dropped,
y=y_train_multi,
cv=cv_strategy,
scoring="f1_macro"
)
learning_curve_data_dict[model_name] = df_lc
except Exception as e:
print(f"Failed for {model_name}: {e}")
Generating learning curve for: Decision Tree - Regularised Generating learning curve for: Decision Tree - FineTuned Generating learning curve for: Random Forest - Regularised Generating learning curve for: Random Forest - FineTuned Generating learning curve for: XGBoost - Regularised Failed for XGBoost - Regularised: 'super' object has no attribute '__sklearn_tags__' Generating learning curve for: XGBoost - FineTuned Failed for XGBoost - FineTuned: 'super' object has no attribute '__sklearn_tags__' Generating learning curve for: LightGBM - Regularised Generating learning curve for: LightGBM - FineTuned Generating learning curve for: Gradient Boosting - Regularised Generating learning curve for: Gradient Boosting - FineTuned Generating learning curve for: Logistic Regression - Regularised Generating learning curve for: Logistic Regression - FineTuned Generating learning curve for: MLP - Regularised Generating learning curve for: MLP - FineTuned
In [180]:
plot_combined_learning_curves(
learning_curve_data_dict=learning_curve_data_dict,
title_prefix=" "
)
In [202]:
# Step 1: Extract fitted models (these MUST be fitted scikit-learn classifiers)
xgb_model = all_models.get("XGBoost - FineTuned")
lgbm_model = all_models.get("LightGBM - FineTuned")
gb_model = all_models.get("Gradient Boosting - FineTuned")
# Step 2: Verify they are valid estimators
print(f"xgb -> {type(xgb_model)}")
print(f"lgbm -> {type(lgbm_model)}")
print(f"gb -> {type(gb_model)}")
xgb -> <class 'xgboost.sklearn.XGBClassifier'> lgbm -> <class 'lightgbm.sklearn.LGBMClassifier'> gb -> <class 'sklearn.ensemble._gb.GradientBoostingClassifier'>
In [204]:
import sklearn
import xgboost
import lightgbm
print("scikit-learn:", sklearn.__version__)
print("xgboost:", xgboost.__version__)
print("lightgbm:", lightgbm.__version__)
scikit-learn: 1.6.1 xgboost: 2.1.1 lightgbm: 4.6.0
In [ ]: